# -*- coding: utf-8 -*-
from jms.aigroup.ai_group.train_sample_new import jms_ai_group__train_sample_new
from jms.aigroup.ai_dwd.yl_threesegcode_pdd_waybill import jms_ai_dwd__yl_threesegcode_pdd_waybill
from jms.dwd.oms.dwd_yl_oms_oms_order_incre_dt import jms_dwd__dwd_yl_oms_oms_order_incre_dt
from jms.dwd.oms.dwd_yl_oms_oms_waybill_incre_dt import jms_dwd__dwd_yl_oms_oms_waybill_incre_dt
from jms.dim.dim_mysql_lmdm_sys_first_code import jms_dim__dim_mysql_lmdm_sys_first_code
from jms.dim.dim_mysql_lmdm_sys_second_code import jms_dim__dim_mysql_lmdm_sys_second_code
from jms.dim.dim_mysql_lmdm_sys_tail_dispatch_area import jms_dim__dim_mysql_lmdm_sys_tail_dispatch_area
from jms.dim.dim_mysql_lmdm_sys_tail_dispatch_area_staff import jms_dim__dim_mysql_lmdm_sys_tail_dispatch_area_staff
from jms.dim.dim_network_whole_massage import jms_dim__dim_network_whole_massage
from jms.dim.lmdm.dim_yl_sys_package_different_base import jms_dim__dim_yl_sys_package_different_base
from jms.dwd.oms.dwd_yl_oms_interceptorpiece_base_dt import jms_dwd__dwd_yl_oms_interceptorpiece_base_dt
from jms.dwd.tab.dwd_tab_barscan_collect_base_dt import jms_dwd__dwd_tab_barscan_collect_base_dt
from jms.dwd.tab.dwd_tab_end_piece_base_dt import jms_dwd__dwd_tab_end_piece_base_dt

from utils.operators.cluster_for_spark_sql_operator import SparkSqlOperator
from jms.time_sensor.time_after_05_30 import time_after_05_30
from datetime import timedelta

jms_ai_group_yl_threesegcode_sign_new = SparkSqlOperator(
    task_id='jms_ai_group_yl_threesegcode_sign_new',
    email=['jarl.huang@jtexpress.com', 'yl_bigdata@yl-scm.com'],
    name='jms_ai_group_yl_threesegcode_sign_new_{{ execution_date | date_add(1) | cst_ds }}',
    email_on_retry=True,
    retries=1,
    sql='jms/aigroup/ai_dwd/yl_threesegcode_sign_new/execute.sql',
    yarn_queue='pyspark',
    driver_memory='4G',
    driver_cores=4,
    executor_memory='16G',
    executor_cores=4,
    pool_slots=2,
    pool='unlimited_pool',
    num_executors=60,  # spark.dynamicAllocation.enabled 为 True 时，num_executors 表示最少 Executor 数
    conf={'spark.executor.memoryOverhead': '4G',
          'spark.dynamicAllocation.enabled': 'true',
          'spark.shuffle.service.enabled': 'true',
          'spark.dynamicAllocation.maxExecutors': 80,
          'spark.sql.sources.partitionOverwriteMode': 'dynamic',
          'spark.dynamicallocation.enabled': 'true',
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 120,
          'spark.sql.shuffle.partitions': 1280,
          'spark.sql.adaptive.enabled': 'true',
          'spark.sql.adaptive.shuffle.targetPostShuffleInputSize': 1258291200,
          'spark.port.maxRetries': 100,
          'spark.executor.extraJavaOptions': '-XX:+UseG1GC -XX:ParallelGCThreads=4'
          },
    hiveconf={'hive.exec.dynamic.partition': 'true',
              'hive.exec.dynamic.partition.mode': 'nonstrict',
              'hive.exec.max.dynamic.partitions.pernode': 5000,
              'hive.exec.max.dynamic.partitions': 30000
              },
    execution_timeout=timedelta(hours=3),
    priority_weight=10,
)

# 设置依赖
jms_ai_group_yl_threesegcode_sign_new << [
    jms_ai_group__train_sample_new,
    jms_ai_dwd__yl_threesegcode_pdd_waybill,
    time_after_05_30,
    jms_dwd__dwd_yl_oms_oms_order_incre_dt,
    jms_dwd__dwd_yl_oms_oms_waybill_incre_dt,
    jms_dim__dim_mysql_lmdm_sys_first_code,
    jms_dim__dim_mysql_lmdm_sys_second_code,
    jms_dim__dim_mysql_lmdm_sys_tail_dispatch_area,
    jms_dim__dim_mysql_lmdm_sys_tail_dispatch_area_staff,
    jms_dim__dim_network_whole_massage,
    jms_dim__dim_yl_sys_package_different_base,
    jms_dwd__dwd_yl_oms_interceptorpiece_base_dt,
    jms_dwd__dwd_tab_barscan_collect_base_dt,
    jms_dwd__dwd_tab_end_piece_base_dt
]
